

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class XvideosWebHtmlProcessor {
	public static void main(String[] args) {
		String player = 
				"<script>\n" +
						"  logged_user = false;\n" + 
						"  var static_id_cdn = 2;\n" + 
						"  var html5player = new HTML5Player('html5video', '30114163');\n" + 
						"  html5player.setVideoTitle('Miki Sato and young boy');\n" + 
						"  html5player.setSponsors(false);\n" + 
						"  html5player.setVideoUrlLow('https://video-hw.xvideos-cdn.com/videos/3gp/f/7/2/xvideos.com_f720aae3c3b5dbb429f4f7cdf4cc62d4.mp4?e=1518886170&ri=1024&rs=85&h=940db0ac5a1eb81f80ccc074442de417');\n" + 
						"  html5player.setVideoUrlHigh('https://video-hw.xvideos-cdn.com/videos/mp4/f/7/2/xvideos.com_f720aae3c3b5dbb429f4f7cdf4cc62d4.mp4?e=1518886170&ri=1024&rs=85&h=8fe24339ff0cd285b4bb9e7e5e02f426');\n" + 
						"  html5player.setVideoHLS('https://hls-hw.xvideos-cdn.com/videos/hls/f7/20/aa/f720aae3c3b5dbb429f4f7cdf4cc62d4/hls.m3u8?e=1518886170&l=0&h=8ce5c79ff5bf74543d91d99d0d7f3f31');\n" + 
						"  html5player.setThumbUrl('https://img-hw.xvideos-cdn.com/videos/thumbslll/f7/20/aa/f720aae3c3b5dbb429f4f7cdf4cc62d4/f720aae3c3b5dbb429f4f7cdf4cc62d4.10.jpg');\n" + 
						"  html5player.setThumbUrl169('https://img-hw.xvideos-cdn.com/videos/thumbs169lll/f7/20/aa/f720aae3c3b5dbb429f4f7cdf4cc62d4/f720aae3c3b5dbb429f4f7cdf4cc62d4.19.jpg');\n" + 
						"  html5player.setRelated(video_related);\n" + 
						"  html5player.setThumbSlide('https://img-hw.xvideos-cdn.com/videos/thumbs169/f7/20/aa/f720aae3c3b5dbb429f4f7cdf4cc62d4/mozaique.jpg');\n" + 
						"  html5player.setThumbSlideBig('https://img-hw.xvideos-cdn.com/videos/thumbs169/f7/20/aa/f720aae3c3b5dbb429f4f7cdf4cc62d4/mozaiquefull.jpg');\n" + 
						"  html5player.setIdCDN('2');\n" + 
						"  html5player.setIdCdnHLS('2');\n" + 
						"  html5player.setFakePlayer(false);\n" + 
						"  html5player.setDesktopiew(true);\n" + 
						"  html5player.setUploaderName('rahulram463');\n" + 
						"  html5player.setVideoURL('/video30114163/miki_sato_and_young_boy');\n" + 
						"  html5player.setStaticDomain('static-hw.xvideos.com');\n" + 
						"  html5player.setHttps();\n" + 
						"  html5player.setCanUseHttps();\n" + 
						"  html5player.initPlayer();\n" + 
						"  document.getElementById('html5video').style.display = '';\n" + 
						" if (!html5player) {  function createRequestObject() { var xhr; try { xhr = new XMLHttpRequest(); } catch (e) { xhr = new ActiveXObject(\"Microsoft.XMLHTTP\"); } return xhr; } var js_error = createRequestObject(); js_error.open('GET', '/html5player/jserror/30114163/2', true); js_error.send(); }\n" + 
						"</script>";
		//html5PlayerRegular(player);
		jsoup("https://www.xvideos.com/video32683409/young_amateur_interracial_couple_passionately_fucking_for_a_half_an_hour");
	}
	/**
	 * @Description: 使用jsoup抓取网页
	 * @param url   
	 * @return void  
	 * @throws
	 * @author JornTang
	 * @date 2018年2月10日
	 */
	public static void jsoup(String url){
		try {
			/*用來封裝要保存的参数*/  
	        Map<String, Object> map = new HashMap<String, Object>();
			 //从一个网站获取和解析一个HTML文档，jsoup的API中有此方法的说明  
	        Document document = Jsoup.connect(url).get();  
	        /*取得script下面的JS变量*/  
	        Elements e = document.getElementsByTag("script").eq(6);
	        /*循环遍历script下面的JS变量*/  
	        for (Element element : e) {  
	              
	            /*取得JS变量数组*/  
	            String[] data = element.data().toString().split("var");  
	              
	            /*取得单个JS变量*/  
	            for(String variable : data){  
	                  
	                /*过滤variable为空的数据*/  
	                if(variable.contains("=")){  
	                      
	                    /*取到满足条件的JS变量*/  
	                    if(variable.contains("option") || variable.contains("config")   
	                            || variable.contains("color") || variable.contains("innerColor")){  
	                          
	                        String[]  kvp = variable.split("=");  
	                          
	                        /*取得JS变量存入map*/  
	                        if(!map.containsKey(kvp[0].trim())) 
	                        	System.err.println(kvp[1].trim().substring(0, kvp[1].trim().length()-1).toString());
	                            map.put(kvp[0].trim(), kvp[1].trim().substring(0, kvp[1].trim().length()-1).toString());  
	                    }  
	                }  
	            }  
	        }
	        System.out.println(document.data().toString());  
	        //获取所有的img标签  
	       // Elements elements = document.getElementsByTag("td");  
//	        for(Element element : elements){  
//	            //获取每个img标签的src属性的内容，即图片地址，加"abs:"表示绝对路径  
//	            String imgSrc = element.outerHtml();  
//	            System.out.println(imgSrc);  
//	        }  
		} catch (Exception e) {
			// TODO: handle exception
		}
	}
	/**
	 * @Description: htmlunit抓取网页
	 * @param url   
	 * @return void  
	 * @throws
	 * @author JornTang
	 * @date 2018年2月10日
	 */
	public static void htmlunit(String url, String type){
		// 1创建WebClient
		WebClient webClient=new WebClient(BrowserVersion.CHROME);
		try {
			// 2 启动JS
	         webClient.getOptions().setJavaScriptEnabled(true);
	        // 3 禁用Css，可避免自动二次請求CSS进行渲染
	         webClient.getOptions().setCssEnabled(false);
	        // 4 启动客戶端重定向
	         webClient.getOptions().setRedirectEnabled(true);
	        // 5 js运行错誤時，是否拋出异常
	         webClient.getOptions().setThrowExceptionOnScriptError(false);
	        // 6 设置超时
	         webClient.getOptions().setTimeout(3*60*1000);
	         //获取网页
	         HtmlPage htmlPage = webClient.getPage(url);
	        // 等待JS驱动dom完成获得还原后的网页
	          webClient.waitForBackgroundJavaScript(10000);
	        // 网页內容
	        Document doc=Jsoup.parse(htmlPage.asXml());
	        System.err.println(doc.toString());
//	        //抓取代理ip
//	        if(ProxyEnum.IP181.getType().equals(type)){
//	        	ip181Analy(doc);
//	        }else if(ProxyEnum.XICIDAILI.getType().equals(type)){
//	        	xicidailiAnaly(doc);
//	        	//递归获取xicidaili免费IP地址
//	        	String proxyUrl = doHandleUrl(url);
//	        	if(StringUtils.isNotEmpty(proxyUrl)){
//	        		htmlunit(proxyUrl, ProxyEnum.XICIDAILI.getType());
//	        	}
//	        }
		} catch (Exception e) {
			e.printStackTrace();
		}finally{
			//关闭windows，释放资源
			if(webClient != null){
				 webClient.closeAllWindows();
			}
		}
	}
	public static void html5PlayerRegular(String html5Player){
		List<String> ls=new ArrayList<String>();
		List<Pattern> patternlist = new ArrayList<Pattern>();
		html5Player = html5Player.replace("\n", "");
		List<Object[]> ret = new ArrayList<Object[]>();
		ret.add(new Object[]{"(?<=html5player.setVideoUrlHigh\\(\\')(.+?)(?=\\'\\))", Pattern.CASE_INSENSITIVE});
		ret.add(new Object[]{"(?<=html5player.setVideoUrlLow\\(\\')(.+?)(?=\\'\\))", Pattern.CASE_INSENSITIVE});
		ret.add(new Object[]{"(?<=html5player.setVideoHLS\\(\\')(.+?)(?=\\'\\))", Pattern.CASE_INSENSITIVE});
		patternlist.add(Pattern.compile("(?<=html5player.setVideoUrlHigh\\(\\')(.+?)(?=\\'\\))", Pattern.CASE_INSENSITIVE));
		patternlist.add(Pattern.compile("(?<=html5player.setVideoUrlLow\\(\\')(.+?)(?=\\'\\))", Pattern.CASE_INSENSITIVE));
		patternlist.add(Pattern.compile("(?<=html5player.setVideoHLS\\(\\')(.+?)(?=\\'\\))", Pattern.CASE_INSENSITIVE));
		for (int i = 0; i < patternlist.size(); i++) {
			Pattern pattern = patternlist.get(i);
			Matcher matcher = pattern.matcher(html5Player);
			while(matcher.find()){
				String str = matcher.group();
				System.err.println(str);
			}
		}
	}
}
